﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Domain;
using Spider.Visited;

namespace Spider.Spiders.Workers
{
    /// <summary>
    /// www.ygdy8.net爬虫
    /// </summary>
    internal class Ygdy8netSprider : Spider
    {
       private IVisited _visited=new DBVisited();
       public override void StartToWork(ToDoData data)
       {
           try
           {
               PageData pageData = new PageData();
               switch (data.DataSource)
               {
                   case PageDataSource.Ygdy8netIndexPage:
                       pageData = GetPageResult(data.PageUrl);
                       break;
                   
               }
               if (pageData != null)
               {
                   PageRepository.AddPage(pageData);
               }
           }
           catch (Exception ex)
           {
               SpiderCaptain.GetInstance().StopToWork(ex.ToString());
               CustomsLog.WriteExceptionErrorLog("QQ爬虫工作异常", ex);
           }

       }

        protected override string GetFinalUrl(string sourceContent, PageDataSource dataSource)
        {
            throw new NotImplementedException();
        }
        /// <summary>
        /// 处理页面
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        private PageData GetPageResult(string url)
        {
            if(_visited.ContainUrl(url))
                return null;
            var html = GetSourceHtml(url,null);
            string[] links=GetLink(html);
            foreach (var link in links)
            {
                if (link.StartsWith("/html/") && !_visited.ContainUrl("http://www.ygdy8.net" + link))
                {
                    try
                    {
                        ToDoData d=new ToDoData();
                        d.DataSource = PageDataSource.Ygdy8netIndexPage;
                        d.PageUrl = "http://www.ygdy8.net" + link;
                        ToDo.AddUrl(d);
                    }
                    catch (Exception)
                    {
                        //重复URL不管
                    }
                   
                }
            }
            PageData page=new PageData()
            {
                Data = html,
                DataSource = PageDataSource.Ygdy8netIndexPage,
                Date = DateTime.Now,
                PageUrl = url
            };
            try
            {
                _visited.AddUrl(url, null);
            }
            catch (Exception)
            {
                //忽略重复地址
            }
           
            return page;
        }
    }
}
